#python
import os, sys, subprocess
from contextlib import closing
#third-party
import cardsharp as cs
#src
from util import create_log_dir, get_threads
from ..recid_rules_flat.load import load_spss
from ..recid_rules_flat.extract import get_max_cycles, get_max_profiles
from ..configuration import db_info, config
from ..util import get_cnxn, check_file, capture_input
[docs]def load_recid_flat(cmd):
args = cmd.args
def _create_txt_files(first=False):
_max_cycles = get_max_cycles(db_info['recid'])
with closing(get_cnxn(db_info['recid']['name'],{'db_info':db_info['recid']})) as cnx:
with closing(cnx.cursor()) as c:
c.execute('SELECT COUNT(*) FROM Subject')
num_subjects = c.fetchall()[0][0]
print "Spawning Flat File load subprocesses..."
reload_flat = capture_input('Reload criminal history and summary temp files?(yes or no)', as_bool=True) if not first else True
reload_demo = capture_input('Reload demographic_flat temp files?(yes or no)', as_bool=True) if not first else True
for x in xrange(int(args.cores)):
args.position = x
step = num_subjects / args.cores
start = step * args.position
_last = False
if args.cores == args.position + 1:
step = step + (num_subjects % args.cores)
cmd.task_queue.put({
'out_dir': check_file(args.out_dir, as_dir = True),
'verbose': args.verbose,
'db_info': db_info['recid'],
'start': start,
'step': step,
'position': str(args.position),
'log_dir': create_log_dir(os.path.join(config.log_dir, 'set_%i' % config.current_set, 'load_recid_flat', str(args.position))),
'threads': args.thread if args.thread else get_threads(num_subjects, _max_cycles),
'follow_period': int(args.follow_period),
'reload_flat':reload_flat,
'reload_demo':reload_demo,
})
print "Start creation of recidivism flat files..."
cmd.start()
cmd.task_queue.join()
cmd.flush()
print "Recidivism flat load subprocess %i complete." % args.position
def _prompt_spss(first=False):
_max_cycles = get_max_cycles(db_info['recid'])
_max_profiles = get_max_profiles(db_info['recid'])
"""@todo: Add sub processing for speedup"""
if first:
print "...loading criminal history summary file into spss..."
load_spss('summary', os.path.join(args.out_dir, 'recid', 'summary'))
print "...loading criminal history flat file into spss..."
load_spss('flat', os.path.join(args.out_dir, 'recid', 'flat'), args.sort, max_cycles = _max_cycles)
print "...loading demographic flat file into spss..."
load_spss('demographic', os.path.join(args.out_dir, 'recid', 'demographic'), True, max_cycles = _max_profiles)
else:
reload_sum = capture_input('Would you like to reload the summary spss file?(yes or no)', as_bool=True)
reload_flat = capture_input('Would you like to reload the criminal history flat file?(yes or no)', as_bool=True)
reload_demo = capture_input('Would you like to reload the demographic flat file?(yes or no)', as_bool=True)
if reload_sum:
print "...loading criminal history summary file into spss..."
load_spss('summary', os.path.join(args.out_dir, 'recid', 'summary'))
if reload_flat:
print "...loading criminal history flat file into spss..."
load_spss('flat', os.path.join(args.out_dir, 'recid', 'flat'),args.sort, max_cycles = _max_cycles)
if reload_demo:
print "...loading demographic flat file into spss..."
load_spss('demographic', os.path.join(args.out_dir, 'recid', 'demographic'), max_cycles = _max_profiles)
cs.wait()
'...finished loading spss files.'
if capture_input('Is this the first time you have created flat files for this project?(yes or no)', as_bool=True):
_create_txt_files(first=True)
_prompt_spss(first=True)
else:
if capture_input('Would you like to reload the temp files?(yes or no)', as_bool=True):
_create_txt_files()
_prompt_spss()
else:
_prompt_spss()